

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta name="Description" content="scikit-learn: machine learning in Python">

  
  <title>Prediction Latency &mdash; scikit-learn 0.22 documentation</title>
  
  <link rel="canonical" href="http://scikit-learn.org/stable/auto_examples/applications/plot_prediction_latency.html" />

  
  <link rel="shortcut icon" href="../../_static/favicon.ico"/>
  

  <link rel="stylesheet" href="../../_static/css/vendor/bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<script id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
<script src="../../_static/jquery.js"></script> 
</head>
<body>
<nav id="navbar" class="sk-docs-navbar navbar navbar-expand-md navbar-light bg-light py-0">
  <div class="container-fluid sk-docs-container px-0">
      <a class="navbar-brand py-0" href="../../index.html">
        <img
          class="sk-brand-img"
          src="../../_static/scikit-learn-logo-small.png"
          alt="logo"/>
      </a>
    <button
      id="sk-navbar-toggler"
      class="navbar-toggler"
      type="button"
      data-toggle="collapse"
      data-target="#navbarSupportedContent"
      aria-controls="navbarSupportedContent"
      aria-expanded="false"
      aria-label="Toggle navigation"
    >
      <span class="navbar-toggler-icon"></span>
    </button>

    <div class="sk-navbar-collapse collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav mr-auto">
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../../install.html">Install</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../../user_guide.html">User Guide</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../../modules/classes.html">API</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../index.html">Examples</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../getting_started.html">Getting Started</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../tutorial/index.html">Tutorial</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../glossary.html">Glossary</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../developers/index.html">Development</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../faq.html">FAQ</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../related_projects.html">Related packages</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../roadmap.html">Roadmap</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../../about.html">About us</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
        </li>
        <li class="nav-item dropdown nav-more-item-dropdown">
          <a class="sk-nav-link nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
              <a class="sk-nav-dropdown-item dropdown-item" href="../../getting_started.html">Getting Started</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../tutorial/index.html">Tutorial</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../glossary.html">Glossary</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../developers/index.html">Development</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../faq.html">FAQ</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../related_projects.html">Related packages</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../roadmap.html">Roadmap</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../../about.html">About us</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
          </div>
        </li>
      </ul>
      <div id="searchbox" role="search">
          <div class="searchformwrapper">
          <form class="search" action="../../search.html" method="get">
            <input class="sk-search-text-input" type="text" name="q" aria-labelledby="searchlabel" />
            <input class="sk-search-text-btn" type="submit" value="Go" />
          </form>
          </div>
      </div>
    </div>
  </div>
</nav>
<div class="d-flex" id="sk-doc-wrapper">
    <input type="checkbox" name="sk-toggle-checkbox" id="sk-toggle-checkbox">
    <label id="sk-sidemenu-toggle" class="sk-btn-toggle-toc btn sk-btn-primary" for="sk-toggle-checkbox">Toggle Menu</label>
    <div id="sk-sidebar-wrapper" class="border-right">
      <div class="sk-sidebar-toc-wrapper">
        <div class="sk-sidebar-toc-logo">
          <a href="../../index.html">
            <img
              class="sk-brand-img"
              src="../../_static/scikit-learn-logo-small.png"
              alt="logo"/>
          </a>
        </div>
        <div class="btn-group w-100 mb-2" role="group" aria-label="rellinks">
            <a href="svm_gui.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="Libsvm GUI">Prev</a><a href="../index.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="Examples">Up</a>
            <a href="plot_out_of_core_classification.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="Out-of-core classification of text documents">Next</a>
        </div>
        <div class="alert alert-danger p-1 mb-2" role="alert">
          <p class="text-center mb-0">
          <strong>scikit-learn 0.22</strong><br/>
          <a href="http://scikit-learn.org/dev/versions.html">Other versions</a>
          </p>
        </div>
        <div class="alert alert-warning p-1 mb-2" role="alert">
          <p class="text-center mb-0">
            Please <a class="font-weight-bold" href="../../about.html#citing-scikit-learn"><string>cite us</string></a> if you use the software.
          </p>
        </div>
          <div class="sk-sidebar-toc">
            <ul>
<li><a class="reference internal" href="#">Prediction Latency</a></li>
</ul>

          </div>
      </div>
    </div>
    <div id="sk-page-content-wrapper">
      <div class="sk-page-content container-fluid body px-md-3" role="main">
        
  <div class="sphx-glr-download-link-note admonition note">
<p class="admonition-title">Note</p>
<p>Click <a class="reference internal" href="#sphx-glr-download-auto-examples-applications-plot-prediction-latency-py"><span class="std std-ref">here</span></a> to download the full example code or to run this example in your browser via Binder</p>
</div>
<div class="sphx-glr-example-title section" id="prediction-latency">
<span id="sphx-glr-auto-examples-applications-plot-prediction-latency-py"></span><h1>Prediction Latency<a class="headerlink" href="#prediction-latency" title="Permalink to this headline">¶</a></h1>
<p>This is an example showing the prediction latency of various scikit-learn
estimators.</p>
<p>The goal is to measure the latency one can expect when doing predictions
either in bulk or atomic (i.e. one by one) mode.</p>
<p>The plots represent the distribution of the prediction latency as a boxplot.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Authors: Eustache Diemert &lt;eustache@diemert.fr&gt;</span>
<span class="c1"># License: BSD 3 clause</span>

<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">defaultdict</span>

<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">gc</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>

<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">StandardScaler</span>
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
<span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">make_regression</span>
<span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">RandomForestRegressor</span>
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">Ridge</span>
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">SGDRegressor</span>
<span class="kn">from</span> <span class="nn">sklearn.svm</span> <span class="kn">import</span> <span class="n">SVR</span>
<span class="kn">from</span> <span class="nn">sklearn.utils</span> <span class="kn">import</span> <span class="n">shuffle</span>


<span class="k">def</span> <span class="nf">_not_in_sphinx</span><span class="p">():</span>
    <span class="c1"># Hack to detect whether we are running by the sphinx builder</span>
    <span class="k">return</span> <span class="s1">&#39;__file__&#39;</span> <span class="ow">in</span> <span class="nb">globals</span><span class="p">()</span>


<span class="k">def</span> <span class="nf">atomic_benchmark_estimator</span><span class="p">(</span><span class="n">estimator</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Measure runtime prediction of each instance.&quot;&quot;&quot;</span>
    <span class="n">n_instances</span> <span class="o">=</span> <span class="n">X_test</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
    <span class="n">runtimes</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n_instances</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_instances</span><span class="p">):</span>
        <span class="n">instance</span> <span class="o">=</span> <span class="n">X_test</span><span class="p">[[</span><span class="n">i</span><span class="p">],</span> <span class="p">:]</span>
        <span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
        <span class="n">estimator</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">instance</span><span class="p">)</span>
        <span class="n">runtimes</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start</span>
    <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;atomic_benchmark runtimes:&quot;</span><span class="p">,</span> <span class="nb">min</span><span class="p">(</span><span class="n">runtimes</span><span class="p">),</span> <span class="n">np</span><span class="o">.</span><span class="n">percentile</span><span class="p">(</span>
            <span class="n">runtimes</span><span class="p">,</span> <span class="mi">50</span><span class="p">),</span> <span class="nb">max</span><span class="p">(</span><span class="n">runtimes</span><span class="p">))</span>
    <span class="k">return</span> <span class="n">runtimes</span>


<span class="k">def</span> <span class="nf">bulk_benchmark_estimator</span><span class="p">(</span><span class="n">estimator</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">n_bulk_repeats</span><span class="p">,</span> <span class="n">verbose</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Measure runtime prediction of the whole input.&quot;&quot;&quot;</span>
    <span class="n">n_instances</span> <span class="o">=</span> <span class="n">X_test</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
    <span class="n">runtimes</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n_bulk_repeats</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_bulk_repeats</span><span class="p">):</span>
        <span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
        <span class="n">estimator</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
        <span class="n">runtimes</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start</span>
    <span class="n">runtimes</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="o">/</span> <span class="nb">float</span><span class="p">(</span><span class="n">n_instances</span><span class="p">),</span> <span class="n">runtimes</span><span class="p">)))</span>
    <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;bulk_benchmark runtimes:&quot;</span><span class="p">,</span> <span class="nb">min</span><span class="p">(</span><span class="n">runtimes</span><span class="p">),</span> <span class="n">np</span><span class="o">.</span><span class="n">percentile</span><span class="p">(</span>
            <span class="n">runtimes</span><span class="p">,</span> <span class="mi">50</span><span class="p">),</span> <span class="nb">max</span><span class="p">(</span><span class="n">runtimes</span><span class="p">))</span>
    <span class="k">return</span> <span class="n">runtimes</span>


<span class="k">def</span> <span class="nf">benchmark_estimator</span><span class="p">(</span><span class="n">estimator</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">n_bulk_repeats</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Measure runtimes of prediction in both atomic and bulk mode.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    estimator : already trained estimator supporting `predict()`</span>
<span class="sd">    X_test : test input</span>
<span class="sd">    n_bulk_repeats : how many times to repeat when evaluating bulk mode</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    atomic_runtimes, bulk_runtimes : a pair of `np.array` which contain the</span>
<span class="sd">    runtimes in seconds.</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">atomic_runtimes</span> <span class="o">=</span> <span class="n">atomic_benchmark_estimator</span><span class="p">(</span><span class="n">estimator</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">verbose</span><span class="p">)</span>
    <span class="n">bulk_runtimes</span> <span class="o">=</span> <span class="n">bulk_benchmark_estimator</span><span class="p">(</span><span class="n">estimator</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">n_bulk_repeats</span><span class="p">,</span>
                                             <span class="n">verbose</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">atomic_runtimes</span><span class="p">,</span> <span class="n">bulk_runtimes</span>


<span class="k">def</span> <span class="nf">generate_dataset</span><span class="p">(</span><span class="n">n_train</span><span class="p">,</span> <span class="n">n_test</span><span class="p">,</span> <span class="n">n_features</span><span class="p">,</span> <span class="n">noise</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Generate a regression dataset with the given parameters.&quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;generating dataset...&quot;</span><span class="p">)</span>

    <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">coef</span> <span class="o">=</span> <span class="n">make_regression</span><span class="p">(</span><span class="n">n_samples</span><span class="o">=</span><span class="n">n_train</span> <span class="o">+</span> <span class="n">n_test</span><span class="p">,</span>
                                 <span class="n">n_features</span><span class="o">=</span><span class="n">n_features</span><span class="p">,</span> <span class="n">noise</span><span class="o">=</span><span class="n">noise</span><span class="p">,</span> <span class="n">coef</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="n">random_seed</span> <span class="o">=</span> <span class="mi">13</span>
    <span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>
        <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">train_size</span><span class="o">=</span><span class="n">n_train</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="n">n_test</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="n">random_seed</span><span class="p">)</span>
    <span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span> <span class="o">=</span> <span class="n">shuffle</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="n">random_seed</span><span class="p">)</span>

    <span class="n">X_scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">()</span>
    <span class="n">X_train</span> <span class="o">=</span> <span class="n">X_scaler</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">X_train</span><span class="p">)</span>
    <span class="n">X_test</span> <span class="o">=</span> <span class="n">X_scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>

    <span class="n">y_scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">()</span>
    <span class="n">y_train</span> <span class="o">=</span> <span class="n">y_scaler</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">y_train</span><span class="p">[:,</span> <span class="kc">None</span><span class="p">])[:,</span> <span class="mi">0</span><span class="p">]</span>
    <span class="n">y_test</span> <span class="o">=</span> <span class="n">y_scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">y_test</span><span class="p">[:,</span> <span class="kc">None</span><span class="p">])[:,</span> <span class="mi">0</span><span class="p">]</span>

    <span class="n">gc</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
    <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ok&quot;</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span>


<span class="k">def</span> <span class="nf">boxplot_runtimes</span><span class="p">(</span><span class="n">runtimes</span><span class="p">,</span> <span class="n">pred_type</span><span class="p">,</span> <span class="n">configuration</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Plot a new `Figure` with boxplots of prediction runtimes.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    runtimes : list of `np.array` of latencies in micro-seconds</span>
<span class="sd">    cls_names : list of estimator class names that generated the runtimes</span>
<span class="sd">    pred_type : &#39;bulk&#39; or &#39;atomic&#39;</span>

<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">fig</span><span class="p">,</span> <span class="n">ax1</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">6</span><span class="p">))</span>
    <span class="n">bp</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">boxplot</span><span class="p">(</span><span class="n">runtimes</span><span class="p">,</span> <span class="p">)</span>

    <span class="n">cls_infos</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;</span><span class="si">%s</span><span class="se">\n</span><span class="s1">(</span><span class="si">%d</span><span class="s1"> </span><span class="si">%s</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span>
                                  <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;complexity_computer&#39;</span><span class="p">](</span>
                                      <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">]),</span>
                                  <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;complexity_label&#39;</span><span class="p">])</span> <span class="k">for</span>
                 <span class="n">estimator_conf</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;estimators&#39;</span><span class="p">]]</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">setp</span><span class="p">(</span><span class="n">ax1</span><span class="p">,</span> <span class="n">xticklabels</span><span class="o">=</span><span class="n">cls_infos</span><span class="p">)</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">setp</span><span class="p">(</span><span class="n">bp</span><span class="p">[</span><span class="s1">&#39;boxes&#39;</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="s1">&#39;black&#39;</span><span class="p">)</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">setp</span><span class="p">(</span><span class="n">bp</span><span class="p">[</span><span class="s1">&#39;whiskers&#39;</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="s1">&#39;black&#39;</span><span class="p">)</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">setp</span><span class="p">(</span><span class="n">bp</span><span class="p">[</span><span class="s1">&#39;fliers&#39;</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="s1">&#39;red&#39;</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s1">&#39;+&#39;</span><span class="p">)</span>

    <span class="n">ax1</span><span class="o">.</span><span class="n">yaxis</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">linestyle</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="n">which</span><span class="o">=</span><span class="s1">&#39;major&#39;</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s1">&#39;lightgrey&#39;</span><span class="p">,</span>
                   <span class="n">alpha</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>

    <span class="n">ax1</span><span class="o">.</span><span class="n">set_axisbelow</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
    <span class="n">ax1</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;Prediction Time per Instance - </span><span class="si">%s</span><span class="s1">, </span><span class="si">%d</span><span class="s1"> feats.&#39;</span> <span class="o">%</span> <span class="p">(</span>
        <span class="n">pred_type</span><span class="o">.</span><span class="n">capitalize</span><span class="p">(),</span>
        <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_features&#39;</span><span class="p">]))</span>
    <span class="n">ax1</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s1">&#39;Prediction Time (us)&#39;</span><span class="p">)</span>

    <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>


<span class="k">def</span> <span class="nf">benchmark</span><span class="p">(</span><span class="n">configuration</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Run the whole benchmark.&quot;&quot;&quot;</span>
    <span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">generate_dataset</span><span class="p">(</span>
        <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_train&#39;</span><span class="p">],</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_test&#39;</span><span class="p">],</span>
        <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_features&#39;</span><span class="p">])</span>

    <span class="n">stats</span> <span class="o">=</span> <span class="p">{}</span>
    <span class="k">for</span> <span class="n">estimator_conf</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;estimators&#39;</span><span class="p">]:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Benchmarking&quot;</span><span class="p">,</span> <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">])</span>
        <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
        <span class="n">gc</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
        <span class="n">a</span><span class="p">,</span> <span class="n">b</span> <span class="o">=</span> <span class="n">benchmark_estimator</span><span class="p">(</span><span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">],</span> <span class="n">X_test</span><span class="p">)</span>
        <span class="n">stats</span><span class="p">[</span><span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;atomic&#39;</span><span class="p">:</span> <span class="n">a</span><span class="p">,</span> <span class="s1">&#39;bulk&#39;</span><span class="p">:</span> <span class="n">b</span><span class="p">}</span>

    <span class="n">cls_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">estimator_conf</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span>
        <span class="s1">&#39;estimators&#39;</span><span class="p">]]</span>
    <span class="n">runtimes</span> <span class="o">=</span> <span class="p">[</span><span class="mf">1e6</span> <span class="o">*</span> <span class="n">stats</span><span class="p">[</span><span class="n">clf_name</span><span class="p">][</span><span class="s1">&#39;atomic&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">clf_name</span> <span class="ow">in</span> <span class="n">cls_names</span><span class="p">]</span>
    <span class="n">boxplot_runtimes</span><span class="p">(</span><span class="n">runtimes</span><span class="p">,</span> <span class="s1">&#39;atomic&#39;</span><span class="p">,</span> <span class="n">configuration</span><span class="p">)</span>
    <span class="n">runtimes</span> <span class="o">=</span> <span class="p">[</span><span class="mf">1e6</span> <span class="o">*</span> <span class="n">stats</span><span class="p">[</span><span class="n">clf_name</span><span class="p">][</span><span class="s1">&#39;bulk&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">clf_name</span> <span class="ow">in</span> <span class="n">cls_names</span><span class="p">]</span>
    <span class="n">boxplot_runtimes</span><span class="p">(</span><span class="n">runtimes</span><span class="p">,</span> <span class="s1">&#39;bulk (</span><span class="si">%d</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_test&#39;</span><span class="p">],</span>
                     <span class="n">configuration</span><span class="p">)</span>


<span class="k">def</span> <span class="nf">n_feature_influence</span><span class="p">(</span><span class="n">estimators</span><span class="p">,</span> <span class="n">n_train</span><span class="p">,</span> <span class="n">n_test</span><span class="p">,</span> <span class="n">n_features</span><span class="p">,</span> <span class="n">percentile</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Estimate influence of the number of features on prediction time.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>

<span class="sd">    estimators : dict of (name (str), estimator) to benchmark</span>
<span class="sd">    n_train : nber of training instances (int)</span>
<span class="sd">    n_test : nber of testing instances (int)</span>
<span class="sd">    n_features : list of feature-space dimensionality to test (int)</span>
<span class="sd">    percentile : percentile at which to measure the speed (int [0-100])</span>

<span class="sd">    Returns:</span>
<span class="sd">    --------</span>

<span class="sd">    percentiles : dict(estimator_name,</span>
<span class="sd">                       dict(n_features, percentile_perf_in_us))</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">percentiles</span> <span class="o">=</span> <span class="n">defaultdict</span><span class="p">(</span><span class="n">defaultdict</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">n_features</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;benchmarking with </span><span class="si">%d</span><span class="s2"> features&quot;</span> <span class="o">%</span> <span class="n">n</span><span class="p">)</span>
        <span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">generate_dataset</span><span class="p">(</span><span class="n">n_train</span><span class="p">,</span> <span class="n">n_test</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">cls_name</span><span class="p">,</span> <span class="n">estimator</span> <span class="ow">in</span> <span class="n">estimators</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="n">estimator</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
            <span class="n">gc</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
            <span class="n">runtimes</span> <span class="o">=</span> <span class="n">bulk_benchmark_estimator</span><span class="p">(</span><span class="n">estimator</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
            <span class="n">percentiles</span><span class="p">[</span><span class="n">cls_name</span><span class="p">][</span><span class="n">n</span><span class="p">]</span> <span class="o">=</span> <span class="mf">1e6</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">percentile</span><span class="p">(</span><span class="n">runtimes</span><span class="p">,</span>
                                                           <span class="n">percentile</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">percentiles</span>


<span class="k">def</span> <span class="nf">plot_n_features_influence</span><span class="p">(</span><span class="n">percentiles</span><span class="p">,</span> <span class="n">percentile</span><span class="p">):</span>
    <span class="n">fig</span><span class="p">,</span> <span class="n">ax1</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">6</span><span class="p">))</span>
    <span class="n">colors</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">]</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">cls_name</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">percentiles</span><span class="o">.</span><span class="n">keys</span><span class="p">()):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">sorted</span><span class="p">([</span><span class="n">n</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">percentiles</span><span class="p">[</span><span class="n">cls_name</span><span class="p">]</span><span class="o">.</span><span class="n">keys</span><span class="p">()]))</span>
        <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">percentiles</span><span class="p">[</span><span class="n">cls_name</span><span class="p">][</span><span class="n">n</span><span class="p">]</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">x</span><span class="p">])</span>
        <span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="n">colors</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="p">)</span>
    <span class="n">ax1</span><span class="o">.</span><span class="n">yaxis</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">linestyle</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="n">which</span><span class="o">=</span><span class="s1">&#39;major&#39;</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s1">&#39;lightgrey&#39;</span><span class="p">,</span>
                   <span class="n">alpha</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
    <span class="n">ax1</span><span class="o">.</span><span class="n">set_axisbelow</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
    <span class="n">ax1</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;Evolution of Prediction Time with #Features&#39;</span><span class="p">)</span>
    <span class="n">ax1</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s1">&#39;#Features&#39;</span><span class="p">)</span>
    <span class="n">ax1</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s1">&#39;Prediction Time at </span><span class="si">%d%%</span><span class="s1">-ile (us)&#39;</span> <span class="o">%</span> <span class="n">percentile</span><span class="p">)</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>


<span class="k">def</span> <span class="nf">benchmark_throughputs</span><span class="p">(</span><span class="n">configuration</span><span class="p">,</span> <span class="n">duration_secs</span><span class="o">=</span><span class="mf">0.1</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;benchmark throughput for different estimators.&quot;&quot;&quot;</span>
    <span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">generate_dataset</span><span class="p">(</span>
        <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_train&#39;</span><span class="p">],</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_test&#39;</span><span class="p">],</span>
        <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_features&#39;</span><span class="p">])</span>
    <span class="n">throughputs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
    <span class="k">for</span> <span class="n">estimator_config</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;estimators&#39;</span><span class="p">]:</span>
        <span class="n">estimator_config</span><span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
        <span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
        <span class="n">n_predictions</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="k">while</span> <span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start_time</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">duration_secs</span><span class="p">:</span>
            <span class="n">estimator_config</span><span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">[[</span><span class="mi">0</span><span class="p">]])</span>
            <span class="n">n_predictions</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="n">throughputs</span><span class="p">[</span><span class="n">estimator_config</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]]</span> <span class="o">=</span> <span class="n">n_predictions</span> <span class="o">/</span> <span class="n">duration_secs</span>
    <span class="k">return</span> <span class="n">throughputs</span>


<span class="k">def</span> <span class="nf">plot_benchmark_throughput</span><span class="p">(</span><span class="n">throughputs</span><span class="p">,</span> <span class="n">configuration</span><span class="p">):</span>
    <span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">6</span><span class="p">))</span>
    <span class="n">colors</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">]</span>
    <span class="n">cls_infos</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;</span><span class="si">%s</span><span class="se">\n</span><span class="s1">(</span><span class="si">%d</span><span class="s1"> </span><span class="si">%s</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span>
                                  <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;complexity_computer&#39;</span><span class="p">](</span>
                                      <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">]),</span>
                                  <span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;complexity_label&#39;</span><span class="p">])</span> <span class="k">for</span>
                 <span class="n">estimator_conf</span> <span class="ow">in</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;estimators&#39;</span><span class="p">]]</span>
    <span class="n">cls_values</span> <span class="o">=</span> <span class="p">[</span><span class="n">throughputs</span><span class="p">[</span><span class="n">estimator_conf</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]]</span> <span class="k">for</span> <span class="n">estimator_conf</span> <span class="ow">in</span>
                  <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;estimators&#39;</span><span class="p">]]</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">bar</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">throughputs</span><span class="p">)),</span> <span class="n">cls_values</span><span class="p">,</span> <span class="n">width</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="n">colors</span><span class="p">)</span>
    <span class="n">ax</span><span class="o">.</span><span class="n">set_xticks</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mf">0.25</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">throughputs</span><span class="p">)</span> <span class="o">-</span> <span class="mf">0.75</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">throughputs</span><span class="p">)))</span>
    <span class="n">ax</span><span class="o">.</span><span class="n">set_xticklabels</span><span class="p">(</span><span class="n">cls_infos</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
    <span class="n">ymax</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">cls_values</span><span class="p">)</span> <span class="o">*</span> <span class="mf">1.2</span>
    <span class="n">ax</span><span class="o">.</span><span class="n">set_ylim</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="n">ymax</span><span class="p">))</span>
    <span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s1">&#39;Throughput (predictions/sec)&#39;</span><span class="p">)</span>
    <span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;Prediction Throughput for different estimators (</span><span class="si">%d</span><span class="s1"> &#39;</span>
                 <span class="s1">&#39;features)&#39;</span> <span class="o">%</span> <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_features&#39;</span><span class="p">])</span>
    <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>


<span class="c1"># #############################################################################</span>
<span class="c1"># Main code</span>

<span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>

<span class="c1"># #############################################################################</span>
<span class="c1"># Benchmark bulk/atomic prediction speed for various regressors</span>
<span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;n_train&#39;</span><span class="p">:</span> <span class="nb">int</span><span class="p">(</span><span class="mf">1e3</span><span class="p">),</span>
    <span class="s1">&#39;n_test&#39;</span><span class="p">:</span> <span class="nb">int</span><span class="p">(</span><span class="mf">1e2</span><span class="p">),</span>
    <span class="s1">&#39;n_features&#39;</span><span class="p">:</span> <span class="nb">int</span><span class="p">(</span><span class="mf">1e2</span><span class="p">),</span>
    <span class="s1">&#39;estimators&#39;</span><span class="p">:</span> <span class="p">[</span>
        <span class="p">{</span><span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="s1">&#39;Linear Model&#39;</span><span class="p">,</span>
         <span class="s1">&#39;instance&#39;</span><span class="p">:</span> <span class="n">SGDRegressor</span><span class="p">(</span><span class="n">penalty</span><span class="o">=</span><span class="s1">&#39;elasticnet&#39;</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span>
                                  <span class="n">l1_ratio</span><span class="o">=</span><span class="mf">0.25</span><span class="p">,</span> <span class="n">tol</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">),</span>
         <span class="s1">&#39;complexity_label&#39;</span><span class="p">:</span> <span class="s1">&#39;non-zero coefficients&#39;</span><span class="p">,</span>
         <span class="s1">&#39;complexity_computer&#39;</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">clf</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">count_nonzero</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">coef_</span><span class="p">)},</span>
        <span class="p">{</span><span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="s1">&#39;RandomForest&#39;</span><span class="p">,</span>
         <span class="s1">&#39;instance&#39;</span><span class="p">:</span> <span class="n">RandomForestRegressor</span><span class="p">(),</span>
         <span class="s1">&#39;complexity_label&#39;</span><span class="p">:</span> <span class="s1">&#39;estimators&#39;</span><span class="p">,</span>
         <span class="s1">&#39;complexity_computer&#39;</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">clf</span><span class="p">:</span> <span class="n">clf</span><span class="o">.</span><span class="n">n_estimators</span><span class="p">},</span>
        <span class="p">{</span><span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="s1">&#39;SVR&#39;</span><span class="p">,</span>
         <span class="s1">&#39;instance&#39;</span><span class="p">:</span> <span class="n">SVR</span><span class="p">(</span><span class="n">kernel</span><span class="o">=</span><span class="s1">&#39;rbf&#39;</span><span class="p">),</span>
         <span class="s1">&#39;complexity_label&#39;</span><span class="p">:</span> <span class="s1">&#39;support vectors&#39;</span><span class="p">,</span>
         <span class="s1">&#39;complexity_computer&#39;</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">clf</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">support_vectors_</span><span class="p">)},</span>
    <span class="p">]</span>
<span class="p">}</span>
<span class="n">benchmark</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>

<span class="c1"># benchmark n_features influence on prediction speed</span>
<span class="n">percentile</span> <span class="o">=</span> <span class="mi">90</span>
<span class="n">percentiles</span> <span class="o">=</span> <span class="n">n_feature_influence</span><span class="p">({</span><span class="s1">&#39;ridge&#39;</span><span class="p">:</span> <span class="n">Ridge</span><span class="p">()},</span>
                                  <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_train&#39;</span><span class="p">],</span>
                                  <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;n_test&#39;</span><span class="p">],</span>
                                  <span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">250</span><span class="p">,</span> <span class="mi">500</span><span class="p">],</span> <span class="n">percentile</span><span class="p">)</span>
<span class="n">plot_n_features_influence</span><span class="p">(</span><span class="n">percentiles</span><span class="p">,</span> <span class="n">percentile</span><span class="p">)</span>

<span class="c1"># benchmark throughput</span>
<span class="n">throughputs</span> <span class="o">=</span> <span class="n">benchmark_throughputs</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
<span class="n">plot_benchmark_throughput</span><span class="p">(</span><span class="n">throughputs</span><span class="p">,</span> <span class="n">configuration</span><span class="p">)</span>

<span class="n">stop_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;example run in </span><span class="si">%.2f</span><span class="s2">s&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">stop_time</span> <span class="o">-</span> <span class="n">start_time</span><span class="p">))</span>
</pre></div>
</div>
<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 0 minutes  0.000 seconds)</p>
<div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-auto-examples-applications-plot-prediction-latency-py">
<div class="binder-badge docutils container">
<a class="reference external image-reference" href="https://mybinder.org/v2/gh/scikit-learn/scikit-learn/0.22.X?urlpath=lab/tree/notebooks/auto_examples/applications/plot_prediction_latency.ipynb"><img alt="https://mybinder.org/badge_logo.svg" src="https://mybinder.org/badge_logo.svg" width="150px" /></a>
</div>
<div class="sphx-glr-download docutils container">
<p><a class="reference download internal" download="" href="../../_downloads/9650dca0d5e6710a7b8cc6e2c9d67f1c/plot_prediction_latency.py"><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Python</span> <span class="pre">source</span> <span class="pre">code:</span> <span class="pre">plot_prediction_latency.py</span></code></a></p>
</div>
<div class="sphx-glr-download docutils container">
<p><a class="reference download internal" download="" href="../../_downloads/5c86b980d7e904e071de33f75f113b96/plot_prediction_latency.ipynb"><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Jupyter</span> <span class="pre">notebook:</span> <span class="pre">plot_prediction_latency.ipynb</span></code></a></p>
</div>
</div>
<p class="sphx-glr-signature"><a class="reference external" href="https://sphinx-gallery.github.io">Gallery generated by Sphinx-Gallery</a></p>
</div>


      </div>
    <div class="container">
      <footer class="sk-content-footer">
            &copy; 2007 - 2019, scikit-learn developers (BSD License).
          <a href="../../_sources/auto_examples/applications/plot_prediction_latency.rst.txt" rel="nofollow">Show this page source</a>
      </footer>
    </div>
  </div>
</div>
<script src="../../_static/js/vendor/bootstrap.min.js"></script>

<script>
    window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
    ga('create', 'UA-22606712-2', 'auto');
    ga('set', 'anonymizeIp', true);
    ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>


<script>
$(document).ready(function() {
    /* Add a [>>>] button on the top-right corner of code samples to hide
     * the >>> and ... prompts and the output and thus make the code
     * copyable. */
    var div = $('.highlight-python .highlight,' +
                '.highlight-python3 .highlight,' +
                '.highlight-pycon .highlight,' +
		'.highlight-default .highlight')
    var pre = div.find('pre');

    // get the styles from the current theme
    pre.parent().parent().css('position', 'relative');
    var hide_text = 'Hide prompts and outputs';
    var show_text = 'Show prompts and outputs';

    // create and add the button to all the code blocks that contain >>>
    div.each(function(index) {
        var jthis = $(this);
        if (jthis.find('.gp').length > 0) {
            var button = $('<span class="copybutton">&gt;&gt;&gt;</span>');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
            jthis.prepend(button);
        }
        // tracebacks (.gt) contain bare text elements that need to be
        // wrapped in a span to work with .nextUntil() (see later)
        jthis.find('pre:has(.gt)').contents().filter(function() {
            return ((this.nodeType == 3) && (this.data.trim().length > 0));
        }).wrap('<span>');
    });

    // define the behavior of the button when it's clicked
    $('.copybutton').click(function(e){
        e.preventDefault();
        var button = $(this);
        if (button.data('hidden') === 'false') {
            // hide the code output
            button.parent().find('.go, .gp, .gt').hide();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden');
            button.css('text-decoration', 'line-through');
            button.attr('title', show_text);
            button.data('hidden', 'true');
        } else {
            // show the code output
            button.parent().find('.go, .gp, .gt').show();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible');
            button.css('text-decoration', 'none');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
        }
    });

	/*** Add permalink buttons next to glossary terms ***/
	$('dl.glossary > dt[id]').append(function() {
		return ('<a class="headerlink" href="#' +
			    this.getAttribute('id') +
			    '" title="Permalink to this term">¶</a>');
	});
  /*** Hide navbar when scrolling down ***/
  // Returns true when headerlink target matches hash in url
  (function() {
    hashTargetOnTop = function() {
        var hash = window.location.hash;
        if ( hash.length < 2 ) { return false; }

        var target = document.getElementById( hash.slice(1) );
        if ( target === null ) { return false; }

        var top = target.getBoundingClientRect().top;
        return (top < 2) && (top > -2);
    };

    // Hide navbar on load if hash target is on top
    var navBar = document.getElementById("navbar");
    var navBarToggler = document.getElementById("sk-navbar-toggler");
    var navBarHeightHidden = "-" + navBar.getBoundingClientRect().height + "px";
    var $window = $(window);

    hideNavBar = function() {
        navBar.style.top = navBarHeightHidden;
    };

    showNavBar = function() {
        navBar.style.top = "0";
    }

    if (hashTargetOnTop()) {
        hideNavBar()
    }

    var prevScrollpos = window.pageYOffset;
    hideOnScroll = function(lastScrollTop) {
        if (($window.width() < 768) && (navBarToggler.getAttribute("aria-expanded") === 'true')) {
            return;
        }
        if (lastScrollTop > 2 && (prevScrollpos <= lastScrollTop) || hashTargetOnTop()){
            hideNavBar()
        } else {
            showNavBar()
        }
        prevScrollpos = lastScrollTop;
    };

    /*** high preformance scroll event listener***/
    var raf = window.requestAnimationFrame ||
        window.webkitRequestAnimationFrame ||
        window.mozRequestAnimationFrame ||
        window.msRequestAnimationFrame ||
        window.oRequestAnimationFrame;
    var lastScrollTop = $window.scrollTop();

    if (raf) {
        loop();
    }

    function loop() {
        var scrollTop = $window.scrollTop();
        if (lastScrollTop === scrollTop) {
            raf(loop);
            return;
        } else {
            lastScrollTop = scrollTop;
            hideOnScroll(lastScrollTop);
            raf(loop);
        }
    }
  })();
});

</script>
    
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js"></script>
    
</body>
</html>